package edu.unc.ils.memai.vocab;

import java.io.File;
import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import maui.stemmers.Stemmer;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.PorterStemFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.QueryParser.Operator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.openrdf.concepts.skos.core.Concept;
import org.openrdf.elmo.ElmoModule;
import org.openrdf.elmo.sesame.SesameManager;
import org.openrdf.elmo.sesame.SesameManagerFactory;
import org.openrdf.repository.Repository;
import org.openrdf.repository.sail.SailRepository;
import org.openrdf.sail.nativerdf.NativeStore;

// The index must match the query model, particularly stemming and stopping
public class LuceneVocabulary extends Vocabulary
{
    private static String CONCEPT_ID_FIELD = "conceptId";
    private static String PREF_LABEL_FIELD = "prefLabel";
    private static String ALT_LABEL_FIELD = "altLabel";
    private static String SCOPE_NOTE_FIELD = "scopeNote";
    
    Directory dir = null;
    Map<String, String> idPrefLabelMap = new HashMap<String, String>();

    List<String> stopwords = new ArrayList<String>();
    
    class MyAnalyzer extends Analyzer {
        
        Stemmer stemmer;
        Set<String> stopwords;
        boolean toLowerCase = true;

        public MyAnalyzer(Stemmer stemmer, List<String> stopwords, boolean toLowerCase)
        {
            this.stemmer = stemmer;
            if (stopwords != null) {
                this.stopwords = new HashSet<String>();
                this.stopwords.addAll(stopwords);
            }
            this.toLowerCase = toLowerCase;
        }
        

        public final TokenStream tokenStream(String fieldName, Reader reader) 
        {
            Tokenizer tok = new StandardTokenizer(Version.LUCENE_34, reader);
            TokenFilter filter = new StandardFilter(Version.LUCENE_34, tok);
            
            if (toLowerCase)
                filter = new LowerCaseFilter(Version.LUCENE_34, filter);
            
            if (stopwords != null)
                filter = new StopFilter(Version.LUCENE_34, filter, stopwords);
            
            if (stemmer != null)
                filter = new PorterStemFilter(filter);
            
            return filter;
            
        }
    }
    
     
    /**
     * Initialize in-memory Lucene index from Sesame
     */
    public void load(String path) throws Exception
    {
        if (debugMode) {
            System.err.println("--- Indexing vocabulary from Sesame...");
        }   
        
        dir = new RAMDirectory();
           
        IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_34, 
                new MyAnalyzer(stemmer, stopwords, toLowerCase));
        
        IndexWriter index = new IndexWriter(dir, iwc);        
        
        // Open the Sesame repository
        String indexes = "spoc,ospc";
        NativeStore store = new NativeStore(new File(path), indexes);
        Repository repository = new SailRepository(store);
        repository.initialize();
        ElmoModule module = new ElmoModule();
        SesameManagerFactory factory = new SesameManagerFactory(module, repository);
        SesameManager manager = factory.createElmoManager(); 
        
        for (Concept concept:  manager.findAll(Concept.class))
        {
            javax.xml.namespace.QName qname = concept.getQName();
            String uri = qname.getNamespaceURI() + qname.getLocalPart();

            Document document = new Document();
            Field cid = new Field(CONCEPT_ID_FIELD, uri, Field.Store.YES, Field.Index.NOT_ANALYZED);
            document.add(cid);
            
            // Pref label
            String prefLabel = concept.getSkosPrefLabel();
            Field fpl = new Field(PREF_LABEL_FIELD, prefLabel, Field.Store.YES, 
                    Field.Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
            fpl.setBoost(2.0F);
            document.add(fpl);
            
            // Add an entry to the pref label map
            idPrefLabelMap.put(uri, prefLabel);
                     
            // Alt labels
            Set<String> altLabels = concept.getSkosAltLabels();
            for (String altLabel: altLabels)
            {
                Field fal = new Field(ALT_LABEL_FIELD, altLabel, Field.Store.YES, Field.Index.ANALYZED, 
                        TermVector.WITH_POSITIONS_OFFSETS);
                document.add(fal);
            }
            
            // Scope notes
            Set<Object> scopeNotes = concept.getSkosScopeNotes();
            for (Object scopeNote: scopeNotes)
            {
                String sn = (String)scopeNote;
                Field fsn = new Field(SCOPE_NOTE_FIELD, sn, Field.Store.NO, Field.Index.ANALYZED, 
                        TermVector.WITH_POSITIONS_OFFSETS);
                document.add(fsn);
            }

            index.addDocument(document);
        }
        repository.shutDown();
        store.shutDown();
        index.commit();
        index.close();
    }
    
    /**
     * Given a term or phrase, return a list of candidates
     */
    public List<String> getSenses(String phrase) throws Exception
    {
        List<String> candidates = new ArrayList<String>();
        
        IndexSearcher searcher = new IndexSearcher(dir);
        MultiFieldQueryParser p = new MultiFieldQueryParser(Version.LUCENE_34,
                new String[] { PREF_LABEL_FIELD, ALT_LABEL_FIELD, SCOPE_NOTE_FIELD }, 
                new MyAnalyzer(stemmer, stopwords, toLowerCase));
        p.setDefaultOperator(Operator.AND);
        Query q =  p.parse(phrase);
        // 95% of words occur in 5 or fewer concepts if scope notes are not included
        // 90% of words occur in 10 or fewer concepts if scope notes are included
        TopDocs topDocs = searcher.search(q, 50);
        ScoreDoc[] hits = topDocs.scoreDocs;
        
        if (hits.length == 0)
            return candidates;

        int c = 0;
        float firstScore = (hits != null) ? hits[0].score : 0F;
        for (ScoreDoc hit: hits)
        {
            float score = hit.score;
            if ((firstScore/score) > 2)
                break;
            
            if (threshold != -1 && c > threshold)
            {
                candidates.clear();
                return candidates;
            }
            
            Document doc = searcher.doc(hit.doc);
            Fieldable cid = doc.getFieldable(CONCEPT_ID_FIELD);
            candidates.add(cid.stringValue());
            
            Fieldable prefLabel = doc.getFieldable(PREF_LABEL_FIELD);
            c++;
        }
        return candidates;
    }
    
    
    public String getTerm(String id)
    {
        return idPrefLabelMap.get(id);
    }
}
